1 module template_processor;
2 import std.typecons:Flag,Yes,No;
3 import std.file;
4 import std.json;
5 import std.path;
6 import std.uni;
7 
8 
9 /** 
10 dub.template.json reference:
11 
12 The params part is checked only once. Keep it at the top of the file.
13 For not conflicting with dub's internal parameters, it uses the syntax #PARAMETER
14 ```json
15  "params": {
16 	"windows": {
17 		//Defines windows specific parameters
18 	},
19 	"linux": {
20 		//Defines linux specific parameters
21 	},
22 	"SOME_GLOBAL_VAR": "This parameter can be used anywhere here by simply using #SOME_GLOBAL_VAR"
23  }
24 ```
25 
26 A dub.template.json can have a parent dub.json(or dub.template.json), this is used for separating some
27 configurations, such as the release one, since things can get hairy quite fast if not done.
28 ```json
29 "$extends": "#HIPREME_ENGINE/dub.json"
30 ```
31 
32 ## Adding the engine optional modules 
33 This can be done by using engineModules property. They will automatically
34 use the absolute path and be added to the linkedDependencies on the current section. It is checked on
35 both root and configurations.
36 
37 Another important feature of it is that the engine distributed modules requires a special distribution of hipengine_api.
38 This distribution is hipengine_api:direct. This module optimizes the function calls to instead of using function pointers,
39 it uses extern definitions, this way, it can be built as a static library.
40 This way, it is checked inside the "release" configuration, for making every of them use the subConfiguration of
41 "direct".
42 
43 ```json
44  "engineModules": [
45 	"util",
46 	"game2d",
47 	"math"
48  ]
49 ```
50 
51 Those in linkedDependencies will automatically be added a linker flag called 
52 /WHOLEARCHIVE:depName for windows on ldc compiler. 
53 Since this is an error prone operation, it may be handled by the templater.
54 Also checked in configurations.
55 ```json
56 "linkedDependencies": {
57 	"someDubDep": {"path": "the/path/to/dep"},
58 	"arsd:anything": "11.0"
59  }
60 ```
61 
62 Those in unnamed dependencies will automatically be added to the "dependencies" section.
63 If the path does not exists, it will be ignored and simply do nothing. Also checked in configurations.
64 ```json
65 "unnamedDependencies": [
66 	"some/path/to/dep"
67 ]
68 ```
69 */
70 
71 
72 enum string templateName = "dub.template.json";
73 
74 private immutable string[] systems = 
75 [
76 	"windows",
77 	"linux"
78 ];
79 
80 private enum VariableType
81 {
82 	_default,
83 	currentSystem,
84 	otherSystem
85 }
86 
87 private VariableType getType(string keyName)
88 {
89 	import std.algorithm.searching : countUntil;
90 	string currentSystem = "unknown";
91 	version(Windows)
92 		currentSystem = "windows";
93 	else version(Posix)
94 		currentSystem = "linux";
95 	
96 	if(keyName == currentSystem)
97 		return VariableType.currentSystem;
98 	else if(systems.countUntil(keyName) != -1)
99 		return VariableType.otherSystem;
100 	return VariableType._default;
101 }
102 
103 bool moduleHasDirect(string moduleName)
104 {
105 	switch(moduleName)
106 	{
107 		case "game2d":return true;
108 		default: return false;
109 	}
110 }
111 
112 
113 /** 
114  * 
115  * Params:
116  *   str = Any string
117  *   start = Where the check will start
118  *   varName = Out variable containing the variable name found
119  * Returns: The index where the search stopped
120  */
121 private long getVariableName(in string str, long start, out string varName)
122 {
123 	assert(str[start] == '#');
124 	long curr = start+1;
125 	while(curr < str.length)
126 	{
127 		char ch = str[curr];
128 		if(!(ch.isNumber || ch.isAlpha || ch == '_'))
129 			break;
130 		curr++;
131 	}
132 	varName = str[start+1..curr];
133 	return curr;
134 }
135 
136 
137 private string processString(JSONValue json, string str)
138 {
139 	import std.exception:enforce;
140 	string returnString;
141 	size_t lastStop = 0;
142 	for(size_t i = 0; i < str.length; i++)
143 	{
144 		if(str[i] == '#')
145 		{
146 			returnString~= str[lastStop..i];
147 			string varName;
148 			i = getVariableName(str, i, varName);
149 			enforce(varName in json["params"], "Variable "~varName~" not found");
150 			returnString~= json["params"][varName].str;
151 			lastStop = i;
152 			i--; //For not updating too much
153 		}
154 	}
155 	if(lastStop != str.length) returnString~= str[lastStop..$];
156 	return returnString;
157 }
158 
159 /** 
160  * 
161  * Params:
162  *   f = The file
163  *   variables = Variables to replace in the #VARIABLE text.
164  * Returns: File with replaced text.
165  */
166 private string processFile(string f, string[string] variables)
167 {
168 	string output = "";
169 	size_t lastStop = 0;
170 	for(size_t i = 0; i < f.length; i++)
171 	{
172 		if(f[i] == '#')
173 		{
174 			output~= f[lastStop..i];
175 			string varName;
176 			i = getVariableName(f, i, varName);
177 			assert(varName in variables, "Variable "~varName~" not found");
178 			output~= variables[varName];
179 			lastStop = i;
180 			i--; //For not updating too much
181 		}
182 	}
183 	if(lastStop != f.length) output~= f[lastStop..$];
184 	return output;
185 }
186 
187 /** 
188  * 
189  * Params:
190  *   json = The parsed dub.template.json
191  * Returns: The variables inside "params".
192  */
193 private string[string] getParamsInTemplate(JSONValue json)
194 {
195 	string[string] variables;
196 	if(const(JSONValue)* params = "params" in json)
197 	{
198 		foreach(key, value; params.object)
199 		{
200 			switch(getType(key))
201 			{
202 				case VariableType.currentSystem:
203 				{
204 					foreach(sysKey, sysValue; value.object)
205 						variables[sysKey] = sysValue.str;
206 					break;
207 				}
208 				case VariableType._default:
209 				{
210 					if((key in variables) is null)
211 						variables[key] = value.str;
212 					break;
213 				}
214 				default:break;
215 			}
216 		}
217 	}
218 	return variables;
219 }
220 
221 private string escapeWindowsSep(string thePath)
222 {
223 	string ret;
224 	foreach(ch; thePath)
225 		if(ch == '\\')
226 			ret~= "\\\\";
227 		else ret~= ch;
228 	return ret;
229 }
230 
231 /** 
232  * Saves the current system variables in the cache.
233  * Saves the default type in the cache too.
234  * Params:
235  *   templatePath = Where the file containing the template json is.
236  *   projectPath = The path where the project is contained. Used for the reserved #PROJECT
237  *	 enginePath = Path where the engine is located. Used for the reserved #HIPREME_ENGINE
238  *   settings = Extra settings that will be processed inside the template.
239  *   extraVariables = Optional variables which are always defined.
240  * Returns: THe resulting string
241  */
242 private string processTemplateImpl(string templatePath, string projectPath, string enginePath, const AdditionalSetting[] settings,
243 in string[string] extraVariables)
244 {
245 	string file = readText(templatePath);
246 	JSONValue json = parseJSON(file);
247 	string[string] variables = getParamsInTemplate(json);
248 	string hipremeEngine = enginePath.absolutePath.escapeWindowsSep;
249 	string project = projectPath.absolutePath.escapeWindowsSep;
250 	if(!("params" in json))
251 		json.object["params"] = emptyObject;
252 	json["params"].object["HIPREME_ENGINE"] = hipremeEngine;
253 	json["params"].object["PROJECT"] = project;
254 	foreach(k, v; extraVariables) json["params"].object[k] = v;
255 
256 
257 	foreach(op; settings)
258 	{
259 		JSONValue inherited = emptyObject;
260 		if(op.name in json)
261 		{
262 			inherited = json;
263 			op.handler(json, emptyObject);
264 		}
265 		if("configurations" in json)
266 		{
267 			foreach(cfg; json["configurations"].array)
268 			{
269 				op.handler(cfg, inherited);
270 				cfg.object.remove(op.name);
271 			}
272 		}
273 		if(op.name in json)
274 		{
275 			json.object.remove(op.name);
276 		}
277 	}
278 	variables["PROJECT"] = projectPath.absolutePath.escapeWindowsSep;
279 	variables["HIPREME_ENGINE"] = hipremeEngine;
280 	foreach(k, v; extraVariables) variables[k] = v;
281 	json.object.remove("params");
282 	json.object.remove("$schema");
283 	file = processFile(json.toPrettyString(JSONOptions.doNotEscapeSlashes), variables);
284 	return file;
285 }
286 
287 
288 private struct AdditionalSetting
289 {
290 	string name;
291 	JSONValue delegate(JSONValue dubFile, JSONValue inherited = emptyObject) handler;
292 	Flag!"configAvailable" config = Yes.configAvailable;
293 }
294 private enum emptyObject = JSONValue(string[string].init);
295 private enum emptyArray = JSONValue(JSONValue[].init);
296 
297 enum TemplateProcessorResult
298 {
299     notFound,
300     invalid,
301     success
302 }
303 
304 JSONValue getDubFromTemplate(string templatePath, string enginePath)
305 {
306 	string out_jsonFile;
307 	if(processTemplate(templatePath, enginePath, out_jsonFile) != TemplateProcessorResult.success)
308 		throw new JSONException("Could not succesfully process template at path "~templatePath);
309 	return parseJSON(out_jsonFile);
310 }
311 
312 /** 
313  * 
314  * Params:
315  *   templatePath = path/to/folder/with/dub.template.json
316  *   enginePath = The engine path which will be used for the configuration engineModules
317  *   templateResult = The resulting string which can be used to cache internally or even save a file.
318  *	 additionalVariables = Additional variables that may come as an always defined. Used internally
319  * Returns: The result of the operation
320  */
321 TemplateProcessorResult processTemplate(string templatePath, string enginePath, out string templateResult,
322 in string[string] additionalVariables = string[string].init)
323 {
324     string processedPath = templatePath;
325     processedPath = processedPath.absolutePath;
326     if(!exists(templatePath))
327 	{
328 		templateResult = "Path received '" ~ templatePath ~"' does not exists";
329 		return TemplateProcessorResult.notFound;
330 	}
331     templatePath = buildPath(templatePath, templateName);
332     if(!exists(templatePath))
333 	{
334 		templateResult = "File "~ templatePath~ " does not exists";
335 		return TemplateProcessorResult.notFound;
336 	}
337     AdditionalSetting[] additionals = [
338 		{"$extends", (JSONValue json, JSONValue inherited)
339 		{
340 			import std.exception:enforce;
341 			if(!("$extends" in json))
342 				return json;
343 			string parentDub = json["$extends"].str;
344 			string[] options = [
345 				parentDub,
346 				buildPath(parentDub, "dub.json"),
347 				buildPath(parentDub, "dub.template.json")
348 			];
349 			string[] excludeKeys = ["configurations", "subPackages"];
350 			JSONValue parentJson;
351 			foreach(i, opt; options)
352 			{
353 				opt = processString(json, opt);
354 				enforce(opt != templatePath, "Parent can't point to itself.");
355 				if(exists(opt))
356 				{
357 					if(i == 2)
358 						parentJson = getDubFromTemplate(opt, enginePath);
359 					else
360 						parentJson = parseJSON(cast(string)read(opt));
361 					break;
362 				}
363 			}
364 			import std.conv:to;
365 			enforce(parentJson != JSONValue.init, "Could not find json in paths "~options.to!string);
366 			foreach(key, value; parentJson.object)
367 			{
368 				import std.algorithm.searching : countUntil;
369 				if(excludeKeys.countUntil(key) == -1)
370 				{
371 					if(!(key in json)) json.object[key] = parentJson[key];
372 					else
373 					{
374 						enforce(parentJson[key].type == json[key].type);
375 						//New values that aren't array or object will be overridden
376 						switch(json[key].type)
377 						{
378 							case JSONType.array:
379 							{
380 								JSONValue[] arr = parentJson[key].array;
381 								foreach(parentValue; arr)
382 									json[key].array ~= parentValue;
383 								break;
384 							}
385 							case JSONType.object:
386 							{
387 								foreach(parentKey, parentValue; parentJson[key].object)
388 								{
389 									if(!(parentKey in json[key]))
390 										json[key].object[parentKey] = parentValue;
391 								}
392 								break;
393 							}
394 							//If both define, child json overrides it.
395 							default: continue;
396 						}
397 					}
398 				}
399 			}
400 
401 			return json;
402 		}, No.configAvailable},
403 		{"engineModules", (JSONValue json, JSONValue inherited) 
404 		{
405 			if("engineModules" in json)
406 			foreach(mod; json["engineModules"].array)
407 			{
408 				if(!("linkedDependencies" in json))
409 					json.object["linkedDependencies"] = emptyObject;
410 				json["linkedDependencies"].object[mod.str] = ["path": buildPath(enginePath, "modules", mod.str)];
411 			}
412 			if(json["name"].str == "release")
413 			{
414 				if(!("subConfigurations" in json))
415 					json["subConfigurations"] = emptyObject;
416 				
417 				static void putDirectSubconfiguration(ref JSONValue input, ref JSONValue fromCfg)
418 				{
419 					if("engineModules" in fromCfg) 
420 					foreach(mod; fromCfg["engineModules"].array) 
421 					{
422 						if(moduleHasDirect(mod.str))
423 							input["subConfigurations"][mod.str] = "direct";
424 					}
425 				}
426 				///Put direct from inherited
427 				putDirectSubconfiguration(json, inherited);
428 				putDirectSubconfiguration(json, json);
429 			}
430 			return json;
431 		}},
432 		{"linkedDependencies", (JSONValue json, JSONValue inherited)
433 		{
434 			if(!("linkedDependencies" in json))
435 				return json;
436 			foreach(key, value; json["linkedDependencies"].object)
437 			{
438 				if(!("dependencies" in json))
439 					json.object["dependencies"] = emptyObject;
440 				if(!("lflags-windows-ldc" in json))
441 					json.object["lflags-windows-ldc"] = emptyArray;
442 				json["dependencies"].object[key] = value;
443 				json["lflags-windows-ldc"].array ~= JSONValue("/WHOLEARCHIVE:"~key);
444 			}
445 			return json;
446 		}},
447 		{"unnamedDependencies", (JSONValue json, JSONValue inherited)
448 		{
449 			if(!("unnamedDependencies" in json))
450 				return json;
451 			foreach(unnamedDep; json["unnamedDependencies"].array)
452 			{
453 				import std.stdio;
454 				import std.exception:enforce;
455 				string endingPath;
456 				JSONValue* subConfiguration;
457 				if(unnamedDep.type == JSONType.object)
458 				{
459 					enforce("path" in unnamedDep, "Unnamed dependencies with type object must contain a \"path\"");
460 					endingPath = unnamedDep["path"].str;
461 					subConfiguration = ("subConfiguration" in unnamedDep);
462 					if(subConfiguration && !("subConfigurations" in json))
463 						json.object["subConfigurations"] = emptyObject;
464 				}
465 				else
466 					endingPath = unnamedDep.str;
467 
468 				endingPath = processString(json, endingPath);
469 				import std.algorithm.searching : find;
470 				
471 				string[] dubPath = find!((string f) => exists(f))(
472 				[
473 					buildPath(processedPath, endingPath, "dub.json"),
474 					buildPath(processedPath, endingPath, "dub.template.json")
475 				]);
476 
477 				if(dubPath.length)
478 				{
479 					if(!("dependencies" in json))
480 						json.object["dependencies"] = emptyObject;
481 					JSONValue dubJson = parseJSON(readText(dubPath[0]));
482 					string packageName = dubJson["name"].str;
483 					enforce(!(packageName in json["dependencies"]), "Package "~packageName~" from path "~endingPath~" is already present in the dependencies.");
484 					json["dependencies"][packageName] = ["path": endingPath];
485 					if(subConfiguration)
486 						json["subConfigurations"].object[packageName] = subConfiguration.str;
487 				}
488 				else
489 					writeln("Warning: Unnamed dependency at path ", endingPath, " not found");
490 			}
491 			return json;
492 		}}
493 	];
494 
495     templateResult = processTemplateImpl(templatePath, processedPath, enginePath, additionals, additionalVariables);
496     return TemplateProcessorResult.success;
497 }